

In [ ]:

    
import sys
sys.path.append('..')
%pylab inline
import pylab as pl
import numpy as np

# Some nice default configuration for plots
pl.rcParams['figure.figsize'] = 10, 7.5
pl.rcParams['axes.grid'] = True
pl.gray()

IPython.parallel



In [ ]:

    
from IPython.parallel import Client
client = Client()



In [ ]:

    
len(client)



In [ ]:

    
%px print("Hello from the cluster engines!")



In [ ]:

    
def where_am_i():
    import os
    import socket
    
    return "In process with pid {0} on host: '{1}'".format(
        os.getpid(), socket.gethostname())



In [ ]:

    
where_am_i()

Direct View



In [ ]:

    
direct_view = client.direct_view()



In [ ]:

    
where_am_i_direct_results = direct_view.apply(where_am_i)
where_am_i_results



In [ ]:

    
where_am_i_direct_results.get()



In [ ]:

    
where_am_i_direct_results.get_dict()

Load Balanced View



In [ ]:

    
lb_view = client.load_balanced_view()



In [ ]:

    
where_am_i_lb_result = lb_view.apply(where_am_i)
where_am_i_lb_result



In [ ]:

    
where_am_i_lb_result.get()

Distributed Grid Search for a Linear Support Vector Machine



In [ ]:

    
import sys; sys.path.append('..')



In [ ]:

    
from tutolib import mmap, model_selection
_ = reload(mmap), reload(model_selection)



In [ ]:

    
from sklearn.datasets import load_digits
from sklearn.preprocessing import MinMaxScaler

digits = load_digits()

X = MinMaxScaler().fit_transform(digits.data)
y = digits.target

digits_cv_split_filenames = mmap.persist_cv_splits('digits_10', X, y, 10)
digits_cv_split_filenames



In [ ]:

    
mmap.warm_mmap_on_cv_splits(client, digits_cv_split_filenames)



In [ ]:

    
from sklearn.svm import LinearSVC
from collections import OrderedDict
import numpy as np

linear_svc_params = OrderedDict((
    ('C', np.logspace(-2, 2, 5)),
))
linear_svc = LinearSVC()



In [ ]:

    
linear_svc_search = model_selection.RandomizedGridSeach(lb_view)

linear_svc_search.launch_for_splits(linear_svc, linear_svc_params, digits_cv_split_filenames)



In [ ]:

    
linear_svc_search



In [ ]:

    
linear_svc_search.boxplot_parameters()

Scaling Non-Linear SVM: Kernel Approximations

Motivation: traditional kernel SVM as in SVC has almost cubic complexity w.r.t. `n_samples`



In [ ]:

    
x = np.linspace(0, int(1e3), 100)

pl.plot(x, x ** 3 / 1e9)
pl.xlabel("Number of training samples")
pl.ylabel("Estimated Convergence Time of SMO (in seconds)")



In [ ]:

    
1e6 ** 3 / 1e9 / 60 / 60 / 24 / 365

Solution: Approximate Kernel SVM with a non-linear Kernel Expansion on a Small Basis + Linear SVC



In [ ]:

    
from sklearn.kernel_approximation import Nystroem
from sklearn.pipeline import Pipeline

nystroem_pipeline = Pipeline([
    ('nystroem', Nystroem()),
    ('clf', LinearSVC()),
])



In [ ]:

    
nystroem_pipeline_params = OrderedDict((
    ('nystroem__n_components', [50, 100, 200]),
    ('nystroem__gamma', np.logspace(-2, 2, 5)),
    ('clf__C', np.logspace(-2, 2, 5)),
))



In [ ]:

    
nystroem_search = model_selection.RandomizedGridSeach(lb_view)
nystroem_search.launch_for_splits(nystroem_pipeline, nystroem_pipeline_params, digits_cv_split_filenames)



In [ ]:

    
nystroem_search



In [ ]:

    
nystroem_search.boxplot_parameters()



In [ ]:

    
client.abort()

A Word of Caution on the Scalability of this Implementation Nystroem method

In this example we used LinearSVC that does not provide a partial_fit method hence require to put the Nystroem expansion of complet dataset in memory. Furthermore the Pipeline object does not optimize the memory usage.

To make this example really scalable we would need to:

Replace LinearSVC with an incremental linear model: Perceptron, PassiveAggressiveClassifier, SGDClassifier
Add support for memory efficient partial_fit to sklearn.pipeline.Pipeline
Change our IPython.parallel based model evaluator to use the partial_fit method with small minibatches in the inner model evaluation function.